# # # # import pandas as pd

# # # # # 机构职能类>>csv_file
# # # # file_path = r'/data/lrs/QAnything/qanything_kernel/raw_data/机构职能类_format2.xlsx'
# # # # df = pd.read_excel(file_path)
# # # # df.rename(columns={'content-by': 'content'}, inplace=True)
# # # # df[['content']].to_csv('/data/lrs/QAnything/qanything_kernel/data/机构职能类.csv', index=False, encoding='utf-8')


# # # import pandas as pd
# # # from html2text import html2text
# # # import re

# # # zsk_data_path = r'/data/lrs/QAnything/qanything_kernel/raw_data/all_vaild_kldata.csv'

# # # df = pd.read_csv(zsk_data_path, delimiter='$')

# # # # 去除 KL_CONTENT_TYPE 列中的缺失值
# # # df.dropna(subset=['KL_CONTENT_TYPE'], inplace=True)

# # # # 获取KL_CONTENT_TYPE列的所有唯一值
# # # content_types = df['KL_CONTENT_TYPE'].unique()

# # # qa_type = '问答类'

# # # CONTENTS = []

# # # # for kb_type in content_types:
# # # #     kb_df = df[df['KL_CONTENT_TYPE'] == kb_type]
# # # #     print(kb_df)

# # # for i in range(len(df)):
# # #     try:
# # #         title = df['KL_CONTENT_TITLE'][i]
# # #         content = html2text(df['KL_CONTENT'][i])
# # #         if df['KL_CONTENT_KEYWORD'][i]:
# # #             kb_content = title + '\n' + '关键词：' + df['KL_CONTENT_KEYWORD'][i] + '\n' + content 
# # #         else:
# # #             kb_content = title + '\n' + content
# # #         kb_content = re.sub(r'\n+', '\n', kb_content)
# # #         CONTENTS.append(kb_content)
# # #     except Exception as e:
# # #         print(f'{i+1}')
# # #         continue


# # # data = {
# # #     'KB_CONTENT': CONTENTS,
# # # }

# # # new_df = pd.DataFrame(data)

# # # new_zsk_path = r'/data/lrs/QAnything/qanything_kernel/raw_data/zsk_data_keywords.csv'

# # # new_df.to_csv(new_zsk_path, index=False, encoding='utf8')   # 共计 23565 条政务知识库



# # import pandas as pd

# # test_path = r'/data/lrs/QAnything/qanything_kernel/test_data/test0305new.xlsx'
# # df = pd.read_excel(test_path)

# # df1 = df.dropna(subset=['问题'])
# # df2 = df1.dropna(subset=['正确答案'])

# # print(len(df2))

# # df2.to_csv('/data/lrs/QAnything/qanything_kernel/test_data/test0305new.csv', index=False, encoding='utf-8')

# # print('数据条数：', len(df2))
 
# # Q, A, R, SOURCE = [], [], [], []
# # for i in range(len(df2)):
# # # for i in range(3):
# #     try:
# #         question = df2['问题'].tolist()[i]
# #         right_answer = df2['正确答案'].tolist()[i]
# #         Q.append(question)
# #         R.append(right_answer)
# #     except Exception as e:
# #         print(f'{i+1}')
# #         continue


# # import requests

# # # 请求的URL
# # url = 'http://172.31.30.3:8555/api/local_doc_qa/local_doc_chat'

# # for i in range(len(Q)):
# #     # 要发送的数据
# #     data = {
# #         "user_id": "lrs123",
# #         "kb_ids": ["KB1c8d65109a4e4b66abd1a90400c5aa0b"],
# #         "question": Q[i],
# #         "history": [],
# #         "rerank": "true",
# #         "streaming": "false"
# #     }

# #     # 发送POST请求
# #     response = requests.post(url, json=data)

# #     # 检查响应
# #     if response.status_code == 200:
# #         # print('请求成功:', response.json())
# #         source_info = ''
# #         ai_response = response.json()['response'][0]
# #         for j in range(len(response.json()['source_documents'])):
# #             source_info += response.json()['source_documents'][j]['kernel'].split('KB_CONTENT: ')[-1] + '\n'
# #         SOURCE.append(source_info)
# #         # logging.info(f'{i+1}条:', ai_response)
# #         print(f'{i+1}条:', ai_response)
# #         A.append(ai_response)
# #     else:
# #         # logging.info('请求失败，状态码:', response.status_code)
# #         print(('请求失败，状态码:', response.status_code))


# # D = {
# #     "问题": Q,
# #     "LLM生成的答案": A,
# #     "正确的答案": R,
# #     "溯源信息": SOURCE
# # }

# # new_df = pd.DataFrame(D)

# # new_test_path = r'/data/lrs/QAnything/qanything_kernel/test_data/test_result.csv'

# # # df2.to_csv(new_test_path, index=False, encoding='utf8')

# # new_df.to_csv(new_test_path, index=False, encoding='utf8')


# # test_path = r'/data/lrs/QAnything/qanything_kernel/test_data/test_result.csv'

# # test_raw_path = r'/data/lrs/QAnything/qanything_kernel/test_data/test0305new.csv'

# # df1 = pd.read_csv(test_path)

# # df2 = pd.read_csv(test_raw_path)

# # df2['溯源信息'] = df1['溯源信息']
# # df2['LLM生成的答案'] = df1['LLM生成的答案']

# # df2.to_csv('/data/lrs/QAnything/qanything_kernel/test_data/test_result_0307.csv', index=False, encoding='utf8')



import pandas as pd
import requests
import json

def load_data(file_path):
    return pd.read_excel(file_path)
    # return pd.read_csv(file_path)

def preprocess_data(df):
    df1 = df.dropna(subset=['问题'])

    df2 = df1.dropna(how='all', subset=['问题', '答案'])
    return df2

def analyze_questions(df):
    Q, R = [], []
    for i in range(len(df)):
        try:
            question = df['问题'].iloc[i]
            right_answer = df['答案'].iloc[i]
            Q.append(question)
            R.append(right_answer)
        except Exception as e:
            print(f'Error at index {i+1}: {e}')
            continue
    return Q, R

def query_responses(questions):
    url = 'http://172.31.30.3:8777/api/local_doc_qa/local_doc_chat'
    A, SOURCE = [], []
    for i, question in enumerate(questions):
        headers = {"content-type": "application/json"}
        data = {
            "user_id": "test001",
            "kb_ids": ["kb7d1c4447bd454140a647a6edc9060e5a"],
            "question": question,
            "history": [],
            "rerank": "true",
            "streaming": False
        }

        response = requests.post(url, json=data)
        if response.status_code == 200:
            decode_response = response.text.encode('ISO-8859-1').decode('utf-8')
            ai_responses = decode_response.strip().split('\n\n')[-1]
            data = json.loads(ai_responses.replace('data: ', ''))
            res = data.get('history', '')[0][-1]
            source_info = '\n'.join([doc['content'] for doc in data.get('source_documents', [])])
            SOURCE.append(source_info)
            print(f'{i+1}条:', res)
            A.append(res)
        else:
            print(('请求失败，状态码:', response.status_code))


    #     response = requests.request("POST", url, json=data, headers=headers).json()
    #     if response["code"] == 200:        
    #         ai_response = response['response']
    #         content = [i["content"] for i in response["source_documents"]]
    #         # source_info = '\n'.join([doc['kernel'].split('KB_CONTENT: ')[-1] for doc in response.json().get('source_documents', [])])
    #         SOURCE.append(content)
    #         print(f'{i+1}条:', ai_response)
    #         A.append(ai_response)
    #     else:
    #         print(('请求失败，状态码:', response.status_code))
    return A, SOURCE

def save_result(df, file_path):
    df.to_csv(file_path, index=False, encoding='utf8')

def main():
    test_path = r'/workspace/qanything_local/qanything_kernel/test_data/testnew0402.xlsx'
    # test_path = r'/workspace/qanything_local/qanything_kernel/test_data/test0305new.csv'
    test_result_path = r'/workspace/qanything_local/qanything_kernel/test_data/test_glm4-2024-04-07.csv'

    df = load_data(test_path)
    df = df[1:]
    
    df_processed = preprocess_data(df)

    Q, R = analyze_questions(df_processed)

    A, SOURCE = query_responses(Q)

    df['LLM生成的答案'] = A
    df['溯源信息'] = SOURCE

    save_result(df, test_result_path)

if __name__ == "__main__":
    main()



